import jieba
from mongoDB import MangoDBConnection


def analyzeArticle(text, count):
    words = jieba.lcut(text)

    hit_stopwords = [line.strip() for line in open('hit_stopwords.txt', 'r').readlines()]
    hit_stopwords += ['\n', ' ', 'a', 'o', '\xa0', 'e', '...']

    for word in words:  # 使用 for 循环遍历每个词语并统计个数
        if len(word) <= 1 or word in hit_stopwords:  # 排除单个字的干扰，使得输出结果为词语
            continue
        else:
            count[word] = count.get(word, 0) + 1  # 如果字典里键为 word 的值存在，则返回键的值并加一，如果不存在键word，则返回0再加上1


if __name__ == '__main__':
    mongoDBClient = MangoDBConnection()
    db = mongoDBClient.myclient['water']
    article = db['article']
    words = db['words']

    articles = article.find({})
    count = {}

    for a in articles:
        analyzeArticle(a['content_text'], count)

    sort_count = []

    count_keys = list(count.keys())  # 将字典的所有键值对转化为列表
    count_keys.sort(key=lambda x: count[x], reverse=True)  # 对列表按照词频从大到小的顺序排序

    for key in count_keys:
        sort_count.append({
            'word': key,
            'count': count[key]
        })

    # print(sort_count)
    words.insert_many(sort_count)
